{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Seldon and TensorFlow Serving MNIST Example\n",
    "\n",
    "This example shows how you can combine Seldon with Tensorflo Serving. We will use a Seldon Tensorflow Serving proxy model image that will forward Seldon internal microservice prediction calls out to a Tensorflow serving server.\n",
    "\n",
    "The example will use the MNIST digit classification task with the example MNIST model.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install seldon-core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import requests\n",
    "from random import randint,random\n",
    "import json\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import sys\n",
    "sys.path.append(\"../../../notebooks\")\n",
    "from visualizer import get_graph\n",
    "from seldon_core.proto import prediction_pb2\n",
    "from seldon_core.proto import prediction_pb2_grpc\n",
    "import grpc\n",
    "import tensorflow as tf\n",
    "from tensorflow.core.framework.tensor_pb2 import TensorProto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_image(arr):\n",
    "    two_d = (np.reshape(arr, (28, 28)) * 255).astype(np.uint8)\n",
    "    plt.imshow(two_d,cmap=plt.cm.gray_r, interpolation='nearest')\n",
    "    return plt\n",
    "\n",
    "def download_mnist():\n",
    "    return input_data.read_data_sets(\"MNIST_data/\", one_hot = True)\n",
    "\n",
    "def rest_predict_request(endpoint,data):\n",
    "    request = {\"data\":{\"ndarray\":data.tolist()}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/predict\",\n",
    "                data={\"json\":json.dumps(request),\"isDefault\":True})\n",
    "    return response.json()   \n",
    "\n",
    "def rest_transform_input_request(endpoint,data):\n",
    "    request = {\"data\":{\"ndarray\":data.tolist()}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/transform-input\",\n",
    "                data={\"json\":json.dumps(request),\"isDefault\":True})\n",
    "    return response.json()   \n",
    "\n",
    "def rest_transform_output_request(endpoint,data):\n",
    "    request = {\"data\":{\"ndarray\":data.tolist()}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/transform-output\",\n",
    "                data={\"json\":json.dumps(request),\"isDefault\":True})\n",
    "    return response.json()   \n",
    "\n",
    "def rest_request_ambassador(deploymentName,namespace,endpoint=\"localhost:8003\",arr=None):\n",
    "    payload = {\"data\":{\"names\":[\"a\",\"b\"],\"tensor\":{\"shape\":[1,784],\"values\":arr.tolist()}}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/seldon/\"+namespace+\"/\"+deploymentName+\"/api/v0.1/predictions\",\n",
    "                json=payload)\n",
    "    print(response.status_code)\n",
    "    print(response.text)\n",
    "\n",
    "def grpc_request_internal(data,endpoint=\"localhost:5000\"):\n",
    "    datadef = prediction_pb2.DefaultData(\n",
    "            tftensor=tf.make_tensor_proto(data)\n",
    "        )\n",
    "\n",
    "    request = prediction_pb2.SeldonMessage(data = datadef)\n",
    "    channel = grpc.insecure_channel(endpoint)\n",
    "    stub = prediction_pb2_grpc.ModelStub(channel)\n",
    "    response = stub.Predict(request=request)\n",
    "    return response\n",
    "\n",
    "\n",
    "def gen_mnist_data(mnist):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(1)\n",
    "    chosen=0\n",
    "    gen_image(batch_xs[chosen]).show()\n",
    "    data = batch_xs[chosen].reshape((1,784))\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-04b637d7613e>:7: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use urllib or similar directly.\n",
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "mnist = download_mnist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create MNIST Model Repository\n",
    "You will need tensorflow installed to run these steps."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Tensorflow MNIST example model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'serving'...\n",
      "remote: Enumerating objects: 36, done.\u001b[K\n",
      "remote: Counting objects: 100% (36/36), done.\u001b[K\n",
      "remote: Compressing objects: 100% (24/24), done.\u001b[K\n",
      "remote: Total 17758 (delta 18), reused 19 (delta 12), pack-reused 17722\u001b[K\n",
      "Receiving objects: 100% (17758/17758), 4.77 MiB | 3.13 MiB/s, done.\n",
      "Resolving deltas: 100% (13224/13224), done.\n",
      "Checking connectivity... done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/tensorflow/serving.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model...\n",
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "Extracting /tmp/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "Extracting /tmp/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting /tmp/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting /tmp/t10k-labels-idx1-ubyte.gz\n",
      "2019-05-03 14:41:16.119184: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "training accuracy 0.9178\n",
      "Done training!\n",
      "Exporting trained model to b'../../../mnist-model/1'\n",
      "Done exporting!\n"
     ]
    }
   ],
   "source": [
    "!cd serving/tensorflow_serving/example && python mnist_saved_model.py --training_iteration=100000 ../../../mnist-model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Copy Model to Google Bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: MODEL_REPOSITORY_BUCKET=gs://seldon-tfserving-store\n"
     ]
    }
   ],
   "source": [
    "# CHANGE THIS TO YOUR OWN CHOSEN GOOGLE BUCKET NAME\n",
    "%env MODEL_REPOSITORY_BUCKET=gs://seldon-tfserving-store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating gs://seldon-tfserving-store/...\r\n"
     ]
    }
   ],
   "source": [
    "!gsutil mb ${MODEL_REPOSITORY_BUCKET}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://mnist-model/1/saved_model.pb [Content-Type=application/octet-stream]...\n",
      "Copying file://mnist-model/1/variables/variables.data-00000-of-00001 [Content-Type=application/octet-stream]...\n",
      "Copying file://mnist-model/1/variables/variables.index [Content-Type=application/octet-stream]...\n",
      "| [3 files][ 48.6 KiB/ 48.6 KiB]                                                \n",
      "Operation completed over 3 objects/48.6 KiB.                                     \n"
     ]
    }
   ],
   "source": [
    "!gsutil cp -r mnist-model ${MODEL_REPOSITORY_BUCKET}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test From GCP Cluster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run MNIST Inference Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "serviceaccount/tiller created\n",
      "clusterrolebinding.rbac.authorization.k8s.io/tiller created\n",
      "$HELM_HOME has been configured at /home/clive/.helm.\n",
      "\n",
      "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n",
      "\n",
      "Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.\n",
      "To prevent this, run `helm init` with the --tiller-tls-verify flag.\n",
      "For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation\n",
      "Happy Helming!\n"
     ]
    }
   ],
   "source": [
    "!kubectl -n kube-system create sa tiller\n",
    "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n",
    "!helm init --service-account tiller"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"tiller-deploy\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"tiller-deploy\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "namespace/seldon created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create namespace seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context \"gke_seldon-demos_europe-west3-c_standard-cluster-1\" modified.\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl config set-context $(kubectl config current-context) --namespace=seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core\n",
      "LAST DEPLOYED: Fri May  3 14:50:05 2019\n",
      "NAMESPACE: seldon-system\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Secret\n",
      "NAME                                   TYPE    DATA  AGE\n",
      "seldon-operator-webhook-server-secret  Opaque  0     2s\n",
      "\n",
      "==> v1beta1/CustomResourceDefinition\n",
      "NAME                                         AGE\n",
      "seldondeployments.machinelearning.seldon.io  2s\n",
      "\n",
      "==> v1/ClusterRole\n",
      "seldon-operator-manager-role  2s\n",
      "\n",
      "==> v1/ClusterRoleBinding\n",
      "NAME                                 AGE\n",
      "seldon-operator-manager-rolebinding  2s\n",
      "\n",
      "==> v1/Service\n",
      "NAME                                        TYPE       CLUSTER-IP     EXTERNAL-IP  PORT(S)  AGE\n",
      "seldon-operator-controller-manager-service  ClusterIP  10.35.248.202  <none>       443/TCP  1s\n",
      "\n",
      "==> v1/StatefulSet\n",
      "NAME                                DESIRED  CURRENT  AGE\n",
      "seldon-operator-controller-manager  1        1        1s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                  READY  STATUS             RESTARTS  AGE\n",
      "seldon-operator-controller-manager-0  0/1    ContainerCreating  0         1s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usageMetrics.enabled=true   --namespace seldon-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "partitioned roll out complete: 1 new pods have been updated...\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/seldon-controller-manager -n seldon-system"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup Ingress\n",
    "Please note: There are reported gRPC issues with ambassador (see https://github.com/SeldonIO/seldon-core/issues/473)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   ambassador\n",
      "LAST DEPLOYED: Fri May  3 14:50:22 2019\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Deployment\n",
      "NAME        DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "ambassador  3        3        3           0          0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                         READY  STATUS             RESTARTS  AGE\n",
      "ambassador-7fb5454fb8-9zk8s  0/1    ContainerCreating  0         0s\n",
      "ambassador-7fb5454fb8-pkqcq  0/1    ContainerCreating  0         0s\n",
      "ambassador-7fb5454fb8-x598p  0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "ambassador  1        0s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME               TYPE          CLUSTER-IP     EXTERNAL-IP  PORT(S)                     AGE\n",
      "ambassador-admins  ClusterIP     10.35.244.151  <none>       8877/TCP                    0s\n",
      "ambassador         LoadBalancer  10.35.244.109  <pending>    80:30617/TCP,443:30073/TCP  0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "Congratuations! You've successfully installed Ambassador.\n",
      "\n",
      "For help, visit our Slack at https://d6e.co/slack or view the documentation online at https://www.getambassador.io.\n",
      "\n",
      "To get the IP address of Ambassador, run the following commands:\n",
      "NOTE: It may take a few minutes for the LoadBalancer IP to be available.\n",
      "     You can watch the status of by running 'kubectl get svc -w  --namespace seldon ambassador'\n",
      "\n",
      "  On GKE/Azure:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace seldon ambassador -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\n",
      "\n",
      "  On AWS:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace seldon ambassador -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\n",
      "\n",
      "  echo http://$SERVICE_IP:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install stable/ambassador --name ambassador --set crds.keep=false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"ambassador\" rollout to finish: 0 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 1 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 2 of 3 updated replicas are available...\n",
      "deployment \"ambassador\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deployment.apps/ambassador"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   tfserving-mnist\n",
      "LAST DEPLOYED: Fri May  3 18:51:07 2019\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1alpha2/SeldonDeployment\n",
      "NAME             AGE\n",
      "tfserving-mnist  1s\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install tfserving-mnist --name tfserving-mnist --namespace seldon --set tfserving.model_base_path=${MODEL_REPOSITORY_BUCKET}/mnist-model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "!helm template tfserving-mnist --name tfserving-mnist --namespace seldon --set tfserving.model_base_path=${MODEL_REPOSITORY_BUCKET}/mnist-model > mnist.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "!sed '1,2d' mnist.json > tmp.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"142pt\" height=\"171pt\"\n",
       " viewBox=\"0.00 0.00 142.00 171.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 167)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-167 138,-167 138,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_0</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-8 8,-155 126,-155 126,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"67\" y=\"-139.8\" font-family=\"Times,serif\" font-size=\"14.00\">predictor&#45;0</text>\n",
       "</g>\n",
       "<!-- tfserving&#45;proxy0 -->\n",
       "<g id=\"node1\" class=\"node\"><title>tfserving&#45;proxy0</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"117.5,-124 16.5,-124 16.5,-88 117.5,-88 117.5,-124\"/>\n",
       "<text text-anchor=\"middle\" x=\"67\" y=\"-102.3\" font-family=\"Times,serif\" font-size=\"14.00\">tfserving&#45;proxy</text>\n",
       "</g>\n",
       "<!-- tfserving&#45;proxy0endpoint -->\n",
       "<g id=\"node2\" class=\"node\"><title>tfserving&#45;proxy0endpoint</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"67\" cy=\"-34\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"67\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">REST</text>\n",
       "</g>\n",
       "<!-- tfserving&#45;proxy0&#45;&gt;tfserving&#45;proxy0endpoint -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>tfserving&#45;proxy0&#45;&gt;tfserving&#45;proxy0endpoint</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M67,-87.6966C67,-79.9827 67,-70.7125 67,-62.1124\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"70.5001,-62.1043 67,-52.1043 63.5001,-62.1044 70.5001,-62.1043\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f73682aa7b8>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_graph(\"tmp.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\r\n",
      "\u001b[04m\u001b[31;01m#\u001b[39;49;00m \u001b[04m\u001b[31;01mS\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mu\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mc\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01m:\u001b[39;49;00m \u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01mf\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mv\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mg\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m_\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01mf\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mv\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mg\u001b[39;49;00m\u001b[04m\u001b[31;01m_\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01my\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m.\u001b[39;49;00m\u001b[04m\u001b[31;01mj\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\r\n",
      "{\r\n",
      "    \u001b[34;01m\"apiVersion\"\u001b[39;49;00m: \u001b[33m\"machinelearning.seldon.io/v1alpha2\"\u001b[39;49;00m,\r\n",
      "    \u001b[34;01m\"kind\"\u001b[39;49;00m: \u001b[33m\"SeldonDeployment\"\u001b[39;49;00m,\r\n",
      "    \u001b[34;01m\"metadata\"\u001b[39;49;00m: {\r\n",
      "        \u001b[34;01m\"labels\"\u001b[39;49;00m: {\r\n",
      "            \u001b[34;01m\"app\"\u001b[39;49;00m: \u001b[33m\"seldon\"\u001b[39;49;00m\r\n",
      "        },\r\n",
      "        \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tfserving-mnist\"\u001b[39;49;00m,\r\n",
      "\t\u001b[34;01m\"namespace\"\u001b[39;49;00m: \u001b[33m\"seldon\"\u001b[39;49;00m\t\r\n",
      "    },\r\n",
      "    \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n",
      "        \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tf-mnist\"\u001b[39;49;00m,\r\n",
      "        \u001b[34;01m\"predictors\"\u001b[39;49;00m: [\r\n",
      "            {\r\n",
      "                \u001b[34;01m\"componentSpecs\"\u001b[39;49;00m: [{\r\n",
      "                    \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n",
      "                        \u001b[34;01m\"containers\"\u001b[39;49;00m: [\r\n",
      "                            {\r\n",
      "                                \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/tfserving-proxy:0.1\"\u001b[39;49;00m,\r\n",
      "                                \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tfserving-proxy\"\u001b[39;49;00m\r\n",
      "                            },\r\n",
      "\t\t\t    {\r\n",
      "\t\t\t\t\u001b[34;01m\"args\"\u001b[39;49;00m: [\r\n",
      "\t\t\t\t    \u001b[33m\"/usr/bin/tensorflow_model_server\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t    \u001b[33m\"--port=7001\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t    \u001b[33m\"--model_name=mnist-model\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t    \u001b[33m\"--model_base_path=gs://seldon-models/tfserving/mnist-model\"\u001b[39;49;00m\r\n",
      "\t\t\t\t],\r\n",
      "\t\t\t\t\u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"gcr.io/kubeflow-images-public/tensorflow-serving-1.7:v20180604-0da89b8a\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mnist-model\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t\u001b[34;01m\"ports\"\u001b[39;49;00m: [\r\n",
      "\t\t\t\t    {\r\n",
      "\t\t\t\t\t\u001b[34;01m\"containerPort\"\u001b[39;49;00m: \u001b[34m7001\u001b[39;49;00m,\r\n",
      "\t\t\t\t\t\u001b[34;01m\"protocol\"\u001b[39;49;00m: \u001b[33m\"TCP\"\u001b[39;49;00m\r\n",
      "\t\t\t\t    }\r\n",
      "\t\t\t\t],\r\n",
      "\t\t\t\t\u001b[34;01m\"resources\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t    \u001b[34;01m\"limits\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t\t\u001b[34;01m\"cpu\"\u001b[39;49;00m: \u001b[33m\"4\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t\t\u001b[34;01m\"memory\"\u001b[39;49;00m: \u001b[33m\"4Gi\"\u001b[39;49;00m\r\n",
      "\t\t\t\t    },\r\n",
      "\t\t\t\t    \u001b[34;01m\"requests\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t\t\u001b[34;01m\"cpu\"\u001b[39;49;00m: \u001b[33m\"1\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t\t\u001b[34;01m\"memory\"\u001b[39;49;00m: \u001b[33m\"1Gi\"\u001b[39;49;00m\r\n",
      "\t\t\t\t    }\r\n",
      "\t\t\t\t},\r\n",
      "\t\t\t\t\u001b[34;01m\"securityContext\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t    \u001b[34;01m\"runAsUser\"\u001b[39;49;00m: \u001b[34m1000\u001b[39;49;00m\r\n",
      "\t\t\t\t}\r\n",
      "\t\t\t    }\r\n",
      "\t\t\t],\r\n",
      "\t\t\t\u001b[34;01m\"terminationGracePeriodSeconds\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m\r\n",
      "\t\t    }\r\n",
      "\t\t}],\r\n",
      "                \u001b[34;01m\"graph\"\u001b[39;49;00m: {\r\n",
      "\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tfserving-proxy\"\u001b[39;49;00m,\r\n",
      "\t\t    \u001b[34;01m\"endpoint\"\u001b[39;49;00m: { \u001b[34;01m\"type\"\u001b[39;49;00m : \u001b[33m\"REST\"\u001b[39;49;00m },\r\n",
      "\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"MODEL\"\u001b[39;49;00m,\r\n",
      "\t\t    \u001b[34;01m\"children\"\u001b[39;49;00m: [],\r\n",
      "\t\t    \u001b[34;01m\"parameters\"\u001b[39;49;00m:\r\n",
      "\t\t    [\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"grpc_endpoint\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"localhost:7001\"\u001b[39;49;00m\r\n",
      "\t\t\t},\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"model_name\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"mnist-model\"\u001b[39;49;00m\r\n",
      "\t\t\t},\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"model_output\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"scores\"\u001b[39;49;00m\r\n",
      "\t\t\t},\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"model_input\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"images\"\u001b[39;49;00m\r\n",
      "\t\t\t},\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"signature_name\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"predict_images\"\u001b[39;49;00m\r\n",
      "\t\t\t}\r\n",
      "\t\t    ]\r\n",
      "\t\t},\r\n",
      "                \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mnist-tfserving\"\u001b[39;49;00m,\r\n",
      "                \u001b[34;01m\"replicas\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m\r\n",
      "            }\r\n",
      "        ]\r\n",
      "    }\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!pygmentize mnist.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Port forward Ambassador**\n",
    "\n",
    "```\n",
    "kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADUxJREFUeJzt3X+oXPWZx/HPR7cRsVWjubrBRJOtsjQETJdJWFEWpRjtEtSCrb1/lBRq4x8RWgmoBKEKLuii7faPpXKzhqbQmi22agTZjUgxW1iLo0g1m1VDzLbZe8m9MZvE+oOY+Owf96Rc450zk5kzcyb3eb8gzMx5zpnz5CSfe2bme+58HRECkM8ZdTcAoB6EH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUn8xyJ0tWLAglixZMshdAqns3btXBw4ccCfr9hR+2zdK+rGkMyX9S0Q8VLb+kiVL1Gw2e9klgBKNRqPjdbt+2W/7TEn/LOmrkpZJGrW9rNvnAzBYvbznXyVpd0TsiYijkrZKurmatgD0Wy/hv0TSH2c83lcs+xTb62w3bTenpqZ62B2AKvUS/tk+VPjM7wdHxFhENCKiMTIy0sPuAFSpl/Dvk7R4xuNFksZ7awfAoPQS/pclXWF7qe15kr4paVs1bQHot66H+iLimO07Jf27pof6NkfEzso6A9BXPY3zR8Rzkp6rqBcAA8TlvUBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyTV0yy9tvdKek/ScUnHIqJRRVNzzVtvvVVav+qqq0rrd911V2n9vvvuO+WegJ7CX7guIg5U8DwABoiX/UBSvYY/JG23/YrtdVU0BGAwen3Zf3VEjNu+SNLztv87InbMXKH4obBOki699NIedwegKj2d+SNivLidlPSUpFWzrDMWEY2IaIyMjPSyOwAV6jr8ts+x/YUT9yWtlvRGVY0B6K9eXvZfLOkp2yee5xcR8W+VdAWg77oOf0TskXRlhb3MWePj46X1Q4cOldYffvjh0vqaNWta1lasWFG6bb+VXaPw5ptvlm47NjZWWl+0aFFXPWEaQ31AUoQfSIrwA0kRfiApwg8kRfiBpKr4rT60sXXr1p62f//990vrhw8f7un5e9FuuO6xxx5rWTt69Gjpti+99FJp/dZbby2toxxnfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IinH+AbjmmmtK65s2bRpQJ9X7+OOPS+vtxvJRH878QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4/wD8Oyzz9bdAvAZnPmBpAg/kBThB5Ii/EBShB9IivADSRF+IKm24/y2N0taI2kyIpYXyy6Q9K+SlkjaK+kbEfF//WtzuB07dqy0fuTIkdJ6RPRUH2anc+9zXSdn/p9KuvGkZfdKeiEirpD0QvEYwGmkbfgjYoekgyctvlnSluL+Fkm3VNwXgD7r9j3/xRExIUnF7UXVtQRgEPr+gZ/tdbabtptTU1P93h2ADnUb/v22F0pScTvZasWIGIuIRkQ0RkZGutwdgKp1G/5tktYW99dKeqaadgAMStvw235C0n9K+mvb+2x/R9JDkq63/bak64vHAE4jbcf5I2K0RekrFfdy2jp48OTBkE/bvn17ad12af2yyy4rrV955ZWl9Tq1+7uhPlzhByRF+IGkCD+QFOEHkiL8QFKEH0iKr+6uwNlnn11av/zyy0vru3fvLq23uzLy/PPPL62frj788MO6W5jTOPMDSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKM81fg+PHjpfV2X93dzurVq7ve9ujRo6X1w4cPl9bbXWMwMTFxyj116u677y6tv/POO6X1+fPnt6xdd911pdsuX768tD4XcOYHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQ8yCmUG41GNJvNge1vWNx2222l9SeffLK0Pm/evNJ62fcFtJs+/IMPPiitn3vuuaX18fHx0vqhQ4dK6/1U9n/7vPPOK912z549pfWyawjq1Gg01Gw2O/q+dM78QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5BU29/nt71Z0hpJkxGxvFh2v6TvSpoqVtsYEc/1q8nT3YYNG0rr27ZtK61/9NFHpfWdO3eeck9VaXedSD+n6G43H8IDDzzQsjY62mrm+Tw6OfP/VNKNsyz/UUSsKP4QfOA00zb8EbFD0sEB9AJggHp5z3+n7d/b3mx7OK91BNBSt+H/iaQvSlohaULSo61WtL3OdtN2c2pqqtVqAAasq/BHxP6IOB4Rn0jaJGlVybpjEdGIiEa7L4MEMDhdhd/2whkPvybpjWraATAonQz1PSHpWkkLbO+T9ANJ19peISkk7ZV0Rx97BNAHbcMfEbMNiD7eh17mrFWrWr4rkiTt2LGjtP7ooy0/Uhl6L774Ysva5ORkT8/99NNPl9aXLVvW0/PPdVzhByRF+IGkCD+QFOEHkiL8QFKEH0iKKbqHwMqVK0vrW7duHVAn1bvnnnta1h555JEBdoKTceYHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQY50dfNRqNultAC5z5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+IKm24be92PZvbO+yvdP294rlF9h+3vbbxe38/reLuSQievqD3nRy5j8maUNEfEnS30pab3uZpHslvRARV0h6oXgM4DTRNvwRMRERrxb335O0S9Ilkm6WtKVYbYukW/rVJIDqndJ7fttLJH1Z0u8kXRwRE9L0DwhJF1XdHID+6Tj8tj8v6VeSvh8RR05hu3W2m7abU1NT3fQIoA86Cr/tz2k6+D+PiF8Xi/fbXljUF0qanG3biBiLiEZENEZGRqroGUAFOvm035Iel7QrIn44o7RN0tri/lpJz1TfHoB+6eSru6+W9C1Jr9t+rVi2UdJDkn5p+zuS/iDp6/1pEXPV9HkFdWkb/oj4raRW/0pfqbYdAIPCFX5AUoQfSIrwA0kRfiApwg8kRfiBpJiiG3110003taxdeOGFpdu+++67pfUdO3aU1pctW1Zaz44zP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kxTg/+uqss85qWTvjjN7OPevXry+tL126tGXthhtu6GnfcwFnfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IinF+1ObBBx8srd9xxx2l9dHR0dL6ypUrT7mnTDjzA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBSbcf5bS+W9DNJfynpE0ljEfFj2/dL+q6kqWLVjRHxXL8axdxz++2391RHbzq5yOeYpA0R8artL0h6xfbzRe1HEfFI/9oD0C9twx8RE5Imivvv2d4l6ZJ+Nwagv07pPb/tJZK+LOl3xaI7bf/e9mbb81tss85203ZzampqtlUA1KDj8Nv+vKRfSfp+RByR9BNJX5S0QtOvDB6dbbuIGIuIRkQ0RkZGKmgZQBU6Cr/tz2k6+D+PiF9LUkTsj4jjEfGJpE2SVvWvTQBVaxt+25b0uKRdEfHDGcsXzljta5LeqL49AP3Syaf9V0v6lqTXbb9WLNsoadT2Ckkhaa+k8t+/BDBUOvm0/7eSPEuJMX3gNMYVfkBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQcEYPbmT0l6X9mLFog6cDAGjg1w9rbsPYl0Vu3quztsojo6PvyBhr+z+zcbkZEo7YGSgxrb8Pal0Rv3aqrN172A0kRfiCpusM/VvP+ywxrb8Pal0Rv3aqlt1rf8wOoT91nfgA1qSX8tm+0/abt3bbvraOHVmzvtf267ddsN2vuZbPtSdtvzFh2ge3nbb9d3M46TVpNvd1v+3+LY/ea7b+vqbfFtn9je5ftnba/Vyyv9diV9FXLcRv4y37bZ0p6S9L1kvZJelnSaET810AbacH2XkmNiKh9TNj230n6k6SfRcTyYtk/SjoYEQ8VPzjnR8Q9Q9Lb/ZL+VPfMzcWEMgtnziwt6RZJ31aNx66kr2+ohuNWx5l/laTdEbEnIo5K2irp5hr6GHoRsUPSwZMW3yxpS3F/i6b/8wxci96GQkRMRMSrxf33JJ2YWbrWY1fSVy3qCP8lkv444/E+DdeU3yFpu+1XbK+ru5lZXFxMm35i+vSLau7nZG1nbh6kk2aWHppj182M11WrI/yzzf4zTEMOV0fE30j6qqT1xctbdKajmZsHZZaZpYdCtzNeV62O8O+TtHjG40WSxmvoY1YRMV7cTkp6SsM3+/D+E5OkFreTNffzZ8M0c/NsM0trCI7dMM14XUf4X5Z0he2ltudJ+qakbTX08Rm2zyk+iJHtcySt1vDNPrxN0tri/lpJz9TYy6cMy8zNrWaWVs3HbthmvK7lIp9iKOOfJJ0paXNE/MPAm5iF7b/S9Nlemp7E9Bd19mb7CUnXavq3vvZL+oGkpyX9UtKlkv4g6esRMfAP3lr0dq2mX7r+eebmE++xB9zbNZL+Q9Lrkj4pFm/U9Pvr2o5dSV+jquG4cYUfkBRX+AFJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSOr/AYHF9AlKAGLaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f73645e6860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "{\n",
      "  \"meta\": {\n",
      "    \"puid\": \"h7n5gedmned98nt4l0s3m71mtl\",\n",
      "    \"tags\": {\n",
      "    },\n",
      "    \"routing\": {\n",
      "    },\n",
      "    \"requestPath\": {\n",
      "      \"tfserving-proxy\": \"seldonio/tfserving-proxy:0.1\"\n",
      "    },\n",
      "    \"metrics\": []\n",
      "  },\n",
      "  \"data\": {\n",
      "    \"names\": [\"t:0\", \"t:1\", \"t:2\", \"t:3\", \"t:4\", \"t:5\", \"t:6\", \"t:7\", \"t:8\", \"t:9\"],\n",
      "    \"tensor\": {\n",
      "      \"shape\": [1, 10],\n",
      "      \"values\": [1.5026679900387307E-10, 7.648504052326643E-10, 2.311240223207278E-6, 5.463987235998502E-6, 0.9917333126068115, 5.872762812941801E-6, 4.898904748529276E-8, 1.9045433873543516E-5, 2.1104186089360155E-5, 0.008212855085730553]\n",
      "    }\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "data = gen_mnist_data(mnist)\n",
    "data = data.reshape((784))\n",
    "rest_request_ambassador(\"tfserving-mnist\",\"seldon\",endpoint=\"localhost:8003\",arr=data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analytics and Load Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core-analytics\n",
      "LAST DEPLOYED: Mon Sep 24 15:52:34 2018\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Service\n",
      "NAME                      TYPE       CLUSTER-IP     EXTERNAL-IP  PORT(S)       AGE\n",
      "alertmanager              ClusterIP  10.19.253.159  <none>       80/TCP        1s\n",
      "grafana-prom              NodePort   10.19.251.82   <none>       80:30690/TCP  1s\n",
      "prometheus-node-exporter  ClusterIP  None           <none>       9100/TCP      1s\n",
      "prometheus-seldon         ClusterIP  10.19.245.167  <none>       80/TCP        1s\n",
      "\n",
      "==> v1/Secret\n",
      "NAME                 TYPE    DATA  AGE\n",
      "grafana-prom-secret  Opaque  1     2s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "prometheus  1        2s\n",
      "\n",
      "==> v1/Job\n",
      "NAME                            DESIRED  SUCCESSFUL  AGE\n",
      "grafana-prom-import-dashboards  1        0           1s\n",
      "\n",
      "==> v1beta1/Deployment\n",
      "NAME                     DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "alertmanager-deployment  1        1        1           0          1s\n",
      "grafana-prom-deployment  1        1        1           0          1s\n",
      "prometheus-deployment    1        1        1           0          1s\n",
      "\n",
      "==> v1beta1/DaemonSet\n",
      "NAME                      DESIRED  CURRENT  READY  UP-TO-DATE  AVAILABLE  NODE SELECTOR  AGE\n",
      "prometheus-node-exporter  1        1        0      1           0          <none>         1s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                      READY  STATUS             RESTARTS  AGE\n",
      "grafana-prom-import-dashboards-wzl55      0/1    ContainerCreating  0         1s\n",
      "alertmanager-deployment-557b99ccf8-m27mm  0/1    ContainerCreating  0         1s\n",
      "grafana-prom-deployment-dd84b7788-gmdkc   0/1    ContainerCreating  0         1s\n",
      "prometheus-node-exporter-pnv9k            0/1    ContainerCreating  0         1s\n",
      "prometheus-deployment-78dd89b44f-7t62q    0/1    ContainerCreating  0         1s\n",
      "\n",
      "==> v1/ConfigMap\n",
      "NAME                       DATA  AGE\n",
      "alertmanager-server-conf   1     2s\n",
      "grafana-import-dashboards  7     2s\n",
      "prometheus-rules           4     2s\n",
      "prometheus-server-conf     1     2s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "prometheus  2s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "prometheus  2s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n",
    "    --set grafana_prom_admin_password=password \\\n",
    "    --set persistence.enabled=false \\\n",
    "    --namespace seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "node \"gke-cluster-2-default-pool-54fbe785-wmcb\" labeled\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') role=locust"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   loadtest\n",
      "LAST DEPLOYED: Mon Sep 24 15:52:43 2018\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/ReplicationController\n",
      "NAME             DESIRED  CURRENT  READY  AGE\n",
      "locust-slave-1   1        1        0      0s\n",
      "locust-master-1  1        1        0      0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME             TYPE      CLUSTER-IP     EXTERNAL-IP  PORT(S)                                       AGE\n",
      "locust-master-1  NodePort  10.19.255.204  <none>       5557:32342/TCP,5558:30493/TCP,8089:32034/TCP  0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                   READY  STATUS             RESTARTS  AGE\n",
      "locust-slave-1-zncg4   0/1    ContainerCreating  0         0s\n",
      "locust-master-1-9z8gm  0/1    ContainerCreating  0         0s\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install seldon-core-loadtesting --name loadtest  \\\n",
    "    --namespace seldon \\\n",
    "    --repo https://storage.googleapis.com/seldon-charts \\\n",
    "    --set locust.script=mnist_rest_locust.py \\\n",
    "    --set locust.host=http://tf-mnist:8000 \\\n",
    "    --set oauth.enabled=false \\\n",
    "    --set locust.hatchRate=1 \\\n",
    "    --set locust.clients=1 \\\n",
    "    --set loadtest.sendFeedback=1 \\\n",
    "    --set locust.minWait=0 \\\n",
    "    --set locust.maxWait=0 \\\n",
    "    --set replicaCount=1 \\\n",
    "    --set data.size=784\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You should port-foward the grafana dashboard\n",
    "\n",
    "```\n",
    "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n",
    "```\n",
    "\n",
    "You can then view an analytics dashboard inside the cluster at http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
